#include "../../include/DictProducer2.h"
#include "../../include/DirScanner.h"

//构造函数，转为英文处理
DictProducer::DictProducer(const string &dir)
{
    DirScanner ds;
    ds(dir);//读英文语料文件夹中所有文件名
    _files = ds.files();//得到所有英文语料文件名
}

DictProducer::DictProducer(const string &dir, SplitTool *splitTool)//构造函数，转为中文处理
{
    DirScanner ds;
    ds(dir);
    _files = ds.files();

    /*    //test :打印_file
          for(auto & elem : _files){
          cout << elem << endl;
          } */

    _splitTool = splitTool;
}

void DictProducer::buildCnDict()
{

    unordered_set<string> CnStopWord = Configuration::getInstance()->getZhStopWordList();

    /*     //test:停止词打印
           for(auto & elem : CnStopWord)
           {
           cout << elem << endl;
           } */

    for(auto & file : _files){
        ifstream ifs(file, std::ios::ate);
        if(!ifs.good()){
            LogError("open %s fail!", file.c_str());
            return;
        }
        LogInfo("read: %s", file.c_str());
        size_t length = ifs.tellg();//得到文件长度
        ifs.seekg(std::ios_base::beg);//返回开头
        char *buff = new char[length + 1];
        ifs.read(buff, length + 1);
        string txt(buff);
        //整篇文章一次性加载进来，然后分词
        vector<string> tmp = _splitTool->cut(txt);
        for(auto & i : tmp){
            /*         if(CnStopWord.count(i)){
                       cout << i << endl;
                       } */

            /* if(i == "一"){ */
            /*     cout << "CnStopWord.count(i) = " << CnStopWord.count(i) << endl; */
            /* } */
            //不是停用词 且 UTF-8字节数为3
            if (!CnStopWord.count(i) && getByteNum_UTF8(i[0]) == 3)
            {
                ++_CnDict[i];
            }
        }
        ifs.close();
    }
}

void DictProducer::buildCnIndex()//中文索引的建立
{
    int i = 0;
    for (auto & elem : _CnDict)
    {
        string word = elem.first;
        size_t charNums = word.size() / getByteNum_UTF8(word[0]);
        for (size_t idx = 0, n = 0; n != charNums;++idx,++n){
            size_t charLen = getByteNum_UTF8(word[idx]);
            string subword = word.substr(idx, charLen);

            _CnIndex[subword].insert(i);
            idx += (charLen - 1);
        }
        ++i;
    }
}

size_t DictProducer::getByteNum_UTF8(const char byte)
{
    int byteNum = 0;
    for (size_t i = 0; i < 6; ++i)
    {
        if (byte & (1 << (7 - i)))
            ++byteNum;
        else
            break;
    }
    return byteNum == 0 ? 1 : byteNum;
}

void DictProducer::storeCnDict(const string & filepath)//将中文词典写入文件
{
    ofstream ofs(filepath);
    for(auto & elem : _CnDict){
        ofs << elem.first << " " << elem.second << endl;
    }
    ofs.close();
}

void DictProducer::storeCnIndex(const string & filepath)//将中文索引写入文件
{
    ofstream ofs(filepath);
    for(auto & record : _CnIndex){
        ofs << record.first << " ";
        for(auto & elem : record.second)
        {
            ofs << elem << " ";
        }
        ofs << endl;
    }
    ofs.close();
}

void DictProducer::buildEnDict()
{
    unordered_set<string> EnStopWord=Configuration::getInstance()->getEnStopWordList();
    for(auto & file : _files)
    {
        ifstream ifs(file);
        if(!ifs.good())
        {
            LogError("open %s fail!", file.c_str());
            return;
        }
        LogInfo("read: %s", file.c_str());
        string line;
        while(getline(ifs,line))
        {
            string word;
            istringstream iss(line);
            while(iss>>word)
            {
                string new_word = dealWord(word,EnStopWord);
                insertWord(new_word);
            }
        }
        ifs.close();
    }
}
string DictProducer::dealWord(string &oldWord,unordered_set<string> & EnStopWord)
{
    for(auto & letter : oldWord)
    {
        if(!isalpha(letter)||(EnStopWord.find(oldWord)!=EnStopWord.end()))
        {
            return string();
        }
        if(isupper(letter))
        {
            letter = tolower(letter);
        }
    }
    return oldWord;
} 

void DictProducer::insertWord(string &newWord)
{
    if(newWord==string())
    {
        return;
    }
    else
    {
        if(1 != newWord.size()) {
            ++_EnDict[newWord];
        }
    }
}
void DictProducer::storeEnDict(const string & filepath)//将中文词典写入文件
{
    ofstream ofs(filepath);
    for(auto & elem : _EnDict){
        ofs << elem.first << " " << elem.second << endl;
    }
    ofs.close();
}
void DictProducer::buildEnIndex()
{
    int i=0;
    for (auto & elem : _EnDict)
    {
        string word = elem.first;
        size_t charNums = word.size() / getByteNum_UTF8(word[0]);
        for (size_t idx = 0, n = 0; n != charNums;++idx,++n){
            size_t charLen = getByteNum_UTF8(word[idx]);
            string subword = word.substr(idx, charLen);

            _EnIndex[subword].insert(i);
            idx += (charLen - 1);
        }
        ++i;
    }
}
void DictProducer::storeEnIndex(const string & filepath)//将英文索引写入文件
{

    ofstream ofs(filepath);
    for(auto & record : _EnIndex){
        ofs << record.first << " ";
        for(auto & elem : record.second)
        {
            ofs << elem << " ";
        }
        ofs << endl;
    }
    ofs.close();
}
void DictProducer::createIndexTable(const string &tablename)
{
    string baseQuery="insert into "+tablename+" (word,dict_id) values";
    string insertQuery = baseQuery;
    if(tablename=="t_en_index")
    {
        for(auto & record : _EnIndex){
            for(auto & index : record.second)
            {
                insertQuery += "('"+record.first+"'"+","+"'"+std::to_string(index+1)+"'),";
            }
            if (!insertQuery.empty()) {
                insertQuery.back() = ';';  // 将字符串的最后一个字符修改为分号
            }
            MySQLConnector::getInstance()->executeQuery(insertQuery);
            insertQuery = baseQuery;
        }
    }
    else if(tablename=="t_cn_index")
    {
        for(auto & record : _CnIndex){
            for(auto & index : record.second)
            {
                insertQuery += "('"+record.first+"'"+","+"'"+std::to_string(index+1)+"'),";
            }
            if (!insertQuery.empty()) {
                insertQuery.back() = ';';  // 将字符串的最后一个字符修改为分号
            }
            MySQLConnector::getInstance()->executeQuery(insertQuery);
            insertQuery = baseQuery;
        }
    }
}
void DictProducer::createDictTable(const string &tablename)
{
    string insertQuery="insert into "+tablename+" (word,times) values";
    if(tablename=="t_en_dict")
    {
        for(auto & record : _EnDict){
            insertQuery += "('"+record.first+"'"+","+"'"+std::to_string(record.second+1)+"'),";
        }
        if (!insertQuery.empty()) {
            insertQuery.back() = ';';  // 将字符串的最后一个字符修改为分号
        }
        MySQLConnector::getInstance()->executeQuery(insertQuery);
    }
    else if(tablename=="t_cn_dict")
    {
        for(auto & record : _CnDict){
            insertQuery += "('"+record.first+"'"+","+"'"+std::to_string(record.second+1)+"'),";
        }
        if (!insertQuery.empty()) {
            insertQuery.back() = ';';  // 将字符串的最后一个字符修改为分号
        }
        MySQLConnector::getInstance()->executeQuery(insertQuery);
    }
}
